{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "神经网络所拟合的函数是高度非凸函数，理想的训练目标是，优化这类函数，达到函数最小值点或接近最小值的极小值点。本节讨论优化器和学习率对训练神经网络的影响。\n",
    "\n",
    "开始介绍正文前，复用上一节的数据处理和神经网络模型代码，如果已阅读完上一节，可以直接阅读正文部分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 加载相关库\n",
    "import os\n",
    "import random\n",
    "import paddle\n",
    "import paddle.fluid as fluid\n",
    "from paddle.fluid.dygraph.nn import Conv2D, Pool2D, FC\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "import gzip\n",
    "import json\n",
    "\n",
    "# 定义数据集读取器\n",
    "def load_data(mode='train'):\n",
    "\n",
    "    # 读取数据文件\n",
    "    datafile = './work/mnist.json.gz'\n",
    "    print('loading mnist dataset from {} ......'.format(datafile))\n",
    "    data = json.load(gzip.open(datafile))\n",
    "    # 读取数据集中的训练集，验证集和测试集\n",
    "    train_set, val_set, eval_set = data\n",
    "\n",
    "    # 数据集相关参数，图片高度IMG_ROWS, 图片宽度IMG_COLS\n",
    "    IMG_ROWS = 28\n",
    "    IMG_COLS = 28\n",
    "    # 根据输入mode参数决定使用训练集，验证集还是测试\n",
    "    if mode == 'train':\n",
    "        imgs = train_set[0]\n",
    "        labels = train_set[1]\n",
    "    elif mode == 'valid':\n",
    "        imgs = val_set[0]\n",
    "        labels = val_set[1]\n",
    "    elif mode == 'eval':\n",
    "        imgs = eval_set[0]\n",
    "        labels = eval_set[1]\n",
    "    # 获得所有图像的数量\n",
    "    imgs_length = len(imgs)\n",
    "    # 验证图像数量和标签数量是否一致\n",
    "    assert len(imgs) == len(labels), \\\n",
    "          \"length of train_imgs({}) should be the same as train_labels({})\".format(\n",
    "                  len(imgs), len(labels))\n",
    "\n",
    "    index_list = list(range(imgs_length))\n",
    "\n",
    "    # 读入数据时用到的batchsize\n",
    "    BATCHSIZE = 100\n",
    "\n",
    "    # 定义数据生成器\n",
    "    def data_generator():\n",
    "        # 训练模式下，打乱训练数据\n",
    "        if mode == 'train':\n",
    "            random.shuffle(index_list)\n",
    "        imgs_list = []\n",
    "        labels_list = []\n",
    "        # 按照索引读取数据\n",
    "        for i in index_list:\n",
    "            # 读取图像和标签，转换其尺寸和类型\n",
    "            img = np.reshape(imgs[i], [1, IMG_ROWS, IMG_COLS]).astype('float32')\n",
    "            label = np.reshape(labels[i], [1]).astype('int64')\n",
    "            imgs_list.append(img) \n",
    "            labels_list.append(label)\n",
    "            # 如果当前数据缓存达到了batch size，就返回一个批次数据\n",
    "            if len(imgs_list) == BATCHSIZE:\n",
    "                yield np.array(imgs_list), np.array(labels_list)\n",
    "                # 清空数据缓存列表\n",
    "                imgs_list = []\n",
    "                labels_list = []\n",
    "\n",
    "        # 如果剩余数据的数目小于BATCHSIZE，\n",
    "        # 则剩余数据一起构成一个大小为len(imgs_list)的mini-batch\n",
    "        if len(imgs_list) > 0:\n",
    "            yield np.array(imgs_list), np.array(labels_list)\n",
    "\n",
    "    return data_generator\n",
    "\n",
    "\n",
    "# 定义模型结构\n",
    "class MNIST(fluid.dygraph.Layer):\n",
    "     def __init__(self, name_scope):\n",
    "         super(MNIST, self).__init__(name_scope)\n",
    "         name_scope = self.full_name()\n",
    "         # 定义卷积层，输出通道20，卷积核大小为5，步长为1，padding为2，使用relu激活函数\n",
    "         self.conv1 = Conv2D(name_scope, num_filters=20, filter_size=5, stride=1, padding=2, act='relu')\n",
    "         # 定义池化层，池化核为2，采用最大池化方式\n",
    "         self.pool1 = Pool2D(name_scope, pool_size=2, pool_stride=2, pool_type='max')\n",
    "         # 定义卷积层，输出通道20，卷积核大小为5，步长为1，padding为2，使用relu激活函数\n",
    "         self.conv2 = Conv2D(name_scope, num_filters=20, filter_size=5, stride=1, padding=2, act='relu')\n",
    "         # 定义池化层，池化核为2，采用最大池化方式\n",
    "         self.pool2 = Pool2D(name_scope, pool_size=2, pool_stride=2, pool_type='max')\n",
    "         # 定义全连接层，输出节点数为10，激活函数使用softmax\n",
    "         self.fc = FC(name_scope, size=10, act='softmax')\n",
    "         \n",
    "    # 定义网络的前向计算过程\n",
    "     def forward(self, inputs):\n",
    "         x = self.conv1(inputs)\n",
    "         x = self.pool1(x)\n",
    "         x = self.conv2(x)\n",
    "         x = self.pool2(x)\n",
    "         x = self.fc(x)\n",
    "         return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## 最优的学习率需要多轮调试\n",
    "\n",
    "\n",
    "\n",
    "在深度学习神经网络模型中，通常使用标准的随机梯度下降算法更新参数，学习率代表参数更新幅度的大小。当学习率最优时，模型的有效容量最大。学习率设置和当前深度学习任务有关，合适的学习率往往需要调参经验和大量的实验，总结来说，学习率选取需要注意以下两点：\n",
    "\n",
    "- 学习率不是越小越好。学习率越小，损失函数的变化速度越慢，意味着我们需要花费更长的时间进行收敛。\n",
    "- 学习率不是越大越好。因为只根据总样本集中的一个批次计算梯度，抽样误差会导致计算出的梯度不是全局最优的方向，且存在波动。同时，在接近最优解时，过大的学习率会导致参数在最优解附近震荡，导致损失难以收敛。\n",
    "\n",
    "\n",
    "<center>\n",
    "\n",
    "<img src=\"https://ai-studio-static-online.cdn.bcebos.com/5b405c28bd7f4119a8837bd71b9b224f46f9c37e2c7244a0a23608b952874464\" width = \"800\" height = \"600\" div align=center />\n",
    "\n",
    "</center>  \n",
    "\n",
    "<center>图1: 不同大小学习率对到达最小值的影响</center>  \n",
    "\n",
    "用户可以用下面代码设置不同的学习率，观察训练损失函数值的变化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading mnist dataset from ./work/mnist.json.gz ......\n",
      "epoch: 0, batch: 0, loss is: [2.6023288]\n",
      "epoch: 0, batch: 200, loss is: [0.51073617]\n",
      "epoch: 0, batch: 400, loss is: [0.33406788]\n",
      "epoch: 1, batch: 0, loss is: [0.37497875]\n",
      "epoch: 1, batch: 200, loss is: [0.25907308]\n",
      "epoch: 1, batch: 400, loss is: [0.19153138]\n",
      "epoch: 2, batch: 0, loss is: [0.21573566]\n",
      "epoch: 2, batch: 200, loss is: [0.24012627]\n",
      "epoch: 2, batch: 400, loss is: [0.23427457]\n",
      "epoch: 3, batch: 0, loss is: [0.07920663]\n",
      "epoch: 3, batch: 200, loss is: [0.10945012]\n",
      "epoch: 3, batch: 400, loss is: [0.16697012]\n",
      "epoch: 4, batch: 0, loss is: [0.10167291]\n",
      "epoch: 4, batch: 200, loss is: [0.13941915]\n",
      "epoch: 4, batch: 400, loss is: [0.04219274]\n"
     ]
    }
   ],
   "source": [
    "#仅优化算法的设置有所差别\n",
    "with fluid.dygraph.guard():\n",
    "    model = MNIST(\"mnist\")\n",
    "    model.train()\n",
    "    #调用加载数据的函数\n",
    "    train_loader = load_data('train')\n",
    "    \n",
    "    #设置不同初始学习率\n",
    "    optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.01)\n",
    "    # optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.001)\n",
    "    #  optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.1)\n",
    "    \n",
    "    EPOCH_NUM = 5\n",
    "    for epoch_id in range(EPOCH_NUM):\n",
    "        for batch_id, data in enumerate(train_loader()):\n",
    "            #准备数据，变得更加简洁\n",
    "            image_data, label_data = data\n",
    "            image = fluid.dygraph.to_variable(image_data)\n",
    "            label = fluid.dygraph.to_variable(label_data)\n",
    "            \n",
    "            #前向计算的过程\n",
    "            predict = model(image)\n",
    "            \n",
    "            #计算损失，取一个批次样本损失的平均值\n",
    "            loss = fluid.layers.cross_entropy(predict, label)\n",
    "            avg_loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            #每训练了100批次的数据，打印下当前Loss的情况\n",
    "            if batch_id % 200 == 0:\n",
    "                print(\"epoch: {}, batch: {}, loss is: {}\".format(epoch_id, batch_id, avg_loss.numpy()))\n",
    "            \n",
    "            #后向传播，更新参数的过程\n",
    "            avg_loss.backward()\n",
    "            optimizer.minimize(avg_loss)\n",
    "            model.clear_gradients()\n",
    "\n",
    "    #保存模型参数\n",
    "    fluid.save_dygraph(model.state_dict(), 'mnist')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "\n",
    "## 选择适合的优化算法训练模型\n",
    "\n",
    "学习率是优化器的一个参数，虽然参数更新都是采用梯度下降算法，但是不同的梯度下降算法影响着神经网络的收敛效果。当随机梯度下降算法[SGD](https://www.paddlepaddle.org.cn/documentation/docs/zh/api_cn/optimizer_cn/SGDOptimizer_cn.html#cn-api-fluid-optimizer-sgdoptimizer)无法满足我们的需求时，可以尝试如下三个思路选取优化器。\n",
    "\n",
    "1. 加入“动量”，参数更新的方向更稳定，比如[Momentum](https://www.paddlepaddle.org.cn/documentation/docs/zh/api_cn/optimizer_cn/MomentumOptimizer_cn.html#cn-api-fluid-optimizer-momentumoptimizer)优化器。每个批次的数据含有抽样误差，导致梯度更新的方向波动较大。如果我们引入物理动量的概念，给梯度下降的过程加入一定的“惯性”累积，就可以减少更新路径上的震荡！即每次更新的梯度由“历史多次梯度的累积方向”和“当次梯度”加权相加得到。历史多次梯度的累积方向往往是从全局视角更正确的方向，这与“惯性”的物理概念很像，也是为何其起名为“Momentum”的原因。类似不同品牌和材质的篮球有一定的重量差别，街头篮球队中的投手（擅长中远距离投篮）喜欢稍重篮球的比例较高。一个很重要的原因是，重的篮球惯性大，更不容易受到手势的小幅变形或风吹的影响。\n",
    "\n",
    "2. 根据不同参数距离最优解的远近，动态调整学习率，比如[AdaGrad](https://www.paddlepaddle.org.cn/documentation/docs/zh/api_cn/optimizer_cn/AdagradOptimizer_cn.html#cn-api-fluid-optimizer-adagradoptimizer)优化器。通过调整学习率的实验可以发现：当某个参数的现值距离最优解较远时（表现为梯度的绝对值较大），我们期望参数更新的步长大一些，以便更快收敛到最优解。当某个参数的现值距离最优解较近时（表现为梯度的绝对值较小），我们期望参数的更新步长小一些，以便更精细的逼近最优解。类似于打高尔夫球，专业运动员第一杆开球时，通常会大力打一个远球，让球尽量落在洞口附近。当第二杆面对离洞口较近的球时，他会更轻柔而细致的推杆，避免将球打飞。与此类似，参数更新的步长应该随着优化过程逐渐减少，减少的程度与当前梯度的大小有关。根据这个思想编写的优化算法称为“AdaGrad”，Ada是Adaptive的缩写，表示“适应环境而变化”的意思。\n",
    "\n",
    "3. 因为上述两个优化思路是正交的，所以可以将两个思路结合起来，这就是当前广泛应用的[Adam](https://www.paddlepaddle.org.cn/documentation/docs/zh/api_cn/optimizer_cn/AdamOptimizer_cn.html#cn-api-fluid-optimizer-adamoptimizer)算法。\n",
    "\n",
    "几种优化算法的代码实现如下，可以选择不同的优化算法训练模型，观察训练时间和损失变化的情况。每种优化算法均有更多的参数设置，详情可查阅飞桨的官方API文档。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading mnist dataset from ./work/mnist.json.gz ......\n",
      "epoch: 0, batch: 0, loss is: [2.3845894]\n"
     ]
    }
   ],
   "source": [
    "#仅优化算法的设置有所差别\n",
    "with fluid.dygraph.guard():\n",
    "    model = MNIST(\"mnist\")\n",
    "    model.train()\n",
    "    #调用加载数据的函数\n",
    "    train_loader = load_data('train')\n",
    "    \n",
    "    #四种优化算法的设置方案，可以逐一尝试效果\n",
    "    optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.01)\n",
    "    #optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=0.01)\n",
    "    #optimizer = fluid.optimizer.AdagradOptimizer(learning_rate=0.01)\n",
    "    #optimizer = fluid.optimizer.AdamOptimizer(learning_rate=0.01)\n",
    "    \n",
    "    EPOCH_NUM = 5\n",
    "    for epoch_id in range(EPOCH_NUM):\n",
    "        for batch_id, data in enumerate(train_loader()):\n",
    "            #准备数据，变得更加简洁\n",
    "            image_data, label_data = data\n",
    "            image = fluid.dygraph.to_variable(image_data)\n",
    "            label = fluid.dygraph.to_variable(label_data)\n",
    "            \n",
    "            #前向计算的过程\n",
    "            predict = model(image)\n",
    "            \n",
    "            #计算损失，取一个批次样本损失的平均值\n",
    "            loss = fluid.layers.cross_entropy(predict, label)\n",
    "            avg_loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            #每训练了100批次的数据，打印下当前Loss的情况\n",
    "            if batch_id % 200 == 0:\n",
    "                print(\"epoch: {}, batch: {}, loss is: {}\".format(epoch_id, batch_id, avg_loss.numpy()))\n",
    "            \n",
    "            #后向传播，更新参数的过程\n",
    "            avg_loss.backward()\n",
    "            optimizer.minimize(avg_loss)\n",
    "            model.clear_gradients()\n",
    "\n",
    "    #保存模型参数\n",
    "    fluid.save_dygraph(model.state_dict(), 'mnist')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PaddlePaddle 1.6.0 (Python 3.5)",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
